{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文件去重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import hashlib\n",
    "import shutil\n",
    "\n",
    "TODO # 待验证\n",
    "\n",
    "def file_hash(file_path):\n",
    "    \"\"\"计算文件的MD5哈希值\"\"\"\n",
    "    hash_md5 = hashlib.md5()\n",
    "    with open(file_path, \"rb\") as f:\n",
    "        for chunk in iter(lambda: f.read(4096), b\"\"):\n",
    "            hash_md5.update(chunk)\n",
    "    return hash_md5.hexdigest()\n",
    "\n",
    "\n",
    "def find_duplicate_files(folder1, folder2, duplicates_folder):\n",
    "    \"\"\"\n",
    "    查找两个文件夹中的重复文件，并将它们移动到指定的重复文件夹中。\n",
    "    \"\"\"\n",
    "    # 用于存储文件路径和哈希值的字典\n",
    "    file_hashes = {}\n",
    "\n",
    "    # 遍历第一个文件夹\n",
    "    for root, dirs, files in os.walk(folder1):\n",
    "        for file in files:\n",
    "            file_path = os.path.join(root, file)\n",
    "            file_hash = file_hash(file_path)\n",
    "            if file_hash in file_hashes:\n",
    "                # 如果哈希值已存在，表示找到重复文件\n",
    "                shutil.move(file_path, os.path.join(duplicates_folder, file))\n",
    "                print(f\"Moved {file_path} to {duplicates_folder}\")\n",
    "            else:\n",
    "                file_hashes[file_hash] = file_path\n",
    "\n",
    "    # 遍历第二个文件夹\n",
    "    for root, dirs, files in os.walk(folder2):\n",
    "        for file in files:\n",
    "            file_path = os.path.join(root, file)\n",
    "            file_hash = file_hash(file_path)\n",
    "            if file_hash in file_hashes and file_hashes[file_hash] != file_path:\n",
    "                # 如果哈希值已存在且不是当前文件本身，表示找到重复文件\n",
    "                shutil.move(file_path, os.path.join(duplicates_folder, file))\n",
    "                print(f\"Moved {file_path} to {duplicates_folder}\")\n",
    "\n",
    "\n",
    "# 使用示例\n",
    "folder1 = \"path/to/your/first/folder\"\n",
    "folder2 = \"path/to/your/second/folder\"\n",
    "duplicates_folder = \"path/to/your/duplicates/folder\"\n",
    "\n",
    "# 确保“重复”文件夹存在\n",
    "if not os.path.exists(duplicates_folder):\n",
    "    os.makedirs(duplicates_folder)\n",
    "\n",
    "find_duplicate_files(folder1, folder2, duplicates_folder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "from moviepy import *\n",
    "from collections import defaultdict\n",
    "\n",
    "def get_video_duration(file_path):\n",
    "    try:\n",
    "        with VideoFileClip(file_path) as video:\n",
    "            duration = video.duration\n",
    "        return round(duration, 2)  # 返回时长，保留两位小数\n",
    "    except Exception as e:\n",
    "        print(f\"Error getting duration for {file_path}: {e}\")\n",
    "        return None\n",
    "\n",
    "def contains_chinese(text):\n",
    "    \"\"\"检查字符串中是否包含汉字\"\"\"\n",
    "    for char in text:\n",
    "        if '\\u4e00' <= char <= '\\u9fff':\n",
    "            return True\n",
    "    return False\n",
    "\n",
    "def find_duplicates(source_dir, extensions=None):\n",
    "    \"\"\"查找重复的视频文件\"\"\"\n",
    "    if extensions is None:\n",
    "        extensions = {'.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv'}\n",
    "    \n",
    "    file_groups = defaultdict(list)\n",
    "    \n",
    "    for root, _, files in os.walk(source_dir):\n",
    "        for file in files:\n",
    "            if os.path.splitext(file)[1].lower() in extensions:\n",
    "                file_path = os.path.join(root, file)\n",
    "                try:\n",
    "                    file_size = os.path.getsize(file_path)\n",
    "                    duration = get_video_duration(file_path)\n",
    "                    if duration is not None:\n",
    "                        key = (file_size, duration)\n",
    "                        file_groups[key].append(file_path)\n",
    "                except OSError as e:\n",
    "                    print(f\"无法访问 {file_path}: {str(e)}\")\n",
    "    \n",
    "    return [group for group in file_groups.values() if len(group) > 1]\n",
    "\n",
    "def select_file_to_keep(file_group):\n",
    "    \"\"\"选择要保留的文件：优先保留带汉字的，其次保留名称较长的\"\"\"\n",
    "    # 首先筛选出带汉字的文件\n",
    "    chinese_files = [f for f in file_group if contains_chinese(os.path.basename(f))]\n",
    "    \n",
    "    if chinese_files:\n",
    "        # 如果有带汉字的文件，从中选择名称最长的\n",
    "        return max(chinese_files, key=lambda x: len(os.path.basename(x)))\n",
    "    else:\n",
    "        # 如果没有带汉字的文件，选择所有文件中名称最长的\n",
    "        return max(file_group, key=lambda x: len(os.path.basename(x)))\n",
    "\n",
    "def safe_move(file_path, target_dir):\n",
    "    \"\"\"安全移动文件，处理文件名冲突\"\"\"\n",
    "    filename = os.path.basename(file_path)\n",
    "    dest_path = os.path.join(target_dir, filename)\n",
    "    \n",
    "    if not os.path.exists(dest_path):\n",
    "        shutil.move(file_path, dest_path)\n",
    "        print(f\"已移动: {file_path} -> {dest_path}\")\n",
    "        return\n",
    "    \n",
    "    base, ext = os.path.splitext(filename)\n",
    "    counter = 1\n",
    "    while True:\n",
    "        new_name = f\"{base}_dup{counter}{ext}\"\n",
    "        new_dest = os.path.join(target_dir, new_name)\n",
    "        if not os.path.exists(new_dest):\n",
    "            shutil.move(file_path, new_dest)\n",
    "            print(f\"已移动（重命名）: {file_path} -> {new_dest}\")\n",
    "            return\n",
    "        counter += 1\n",
    "\n",
    "def main():\n",
    "    # 配置参数\n",
    "    source_dir = \"\"      # 要检查的源目录\n",
    "    target_dir = \"\"  # 存放重复文件的目录\n",
    "    \n",
    "    # 确保目标目录存在\n",
    "    os.makedirs(target_dir, exist_ok=True)\n",
    "    \n",
    "    # 查找重复项\n",
    "    duplicates = find_duplicates(source_dir)\n",
    "    \n",
    "    if not duplicates:\n",
    "        print(\"未找到重复文件\")\n",
    "        return\n",
    "    \n",
    "    # 处理重复文件\n",
    "    total_moved = 0\n",
    "    for group in duplicates:\n",
    "        print(f\"\\n发现重复文件组（{len(group)} 个文件）:\")\n",
    "        for file_path in group:\n",
    "            print(f\"  - {file_path}\")\n",
    "        \n",
    "        # 选择要保留的文件\n",
    "        file_to_keep = select_file_to_keep(group)\n",
    "        print(f\"将保留: {file_to_keep}\")\n",
    "        \n",
    "        # 移动其他重复文件\n",
    "        for file_path in group:\n",
    "            if file_path != file_to_keep:\n",
    "                safe_move(file_path, target_dir)\n",
    "                total_moved += 1\n",
    "    \n",
    "    print(f\"\\n完成！共移动 {total_moved} 个重复文件到 {target_dir}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 根据时长和大小去重"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "from moviepy import *\n",
    "from collections import defaultdict\n",
    "\n",
    "# 定义一个函数，用于获取视频时长\n",
    "def get_video_duration(file_path):\n",
    "    try:\n",
    "        with VideoFileClip(file_path) as video:\n",
    "            duration = video.duration\n",
    "        return round(duration, 2)  # 返回时长，保留两位小数\n",
    "    except Exception as e:\n",
    "        print(f\"Error getting duration for {file_path}: {e}\")\n",
    "        return None\n",
    "\n",
    "def find_duplicates(source_dir, extensions=None):\n",
    "    \"\"\"查找重复的视频文件\"\"\"\n",
    "    if extensions is None:\n",
    "        extensions = {'.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv'}\n",
    "    \n",
    "    file_groups = defaultdict(list)\n",
    "    \n",
    "    for root, _, files in os.walk(source_dir):\n",
    "        for file in files:\n",
    "            if os.path.splitext(file)[1].lower() in extensions:\n",
    "                file_path = os.path.join(root, file)\n",
    "                try:\n",
    "                    file_size = os.path.getsize(file_path)\n",
    "                    duration = get_video_duration(file_path)\n",
    "                    if duration is not None:\n",
    "                        key = (file_size, duration)\n",
    "                        file_groups[key].append(file_path)\n",
    "                except OSError as e:\n",
    "                    print(f\"无法访问 {file_path}: {str(e)}\")\n",
    "    \n",
    "    return [group for group in file_groups.values() if len(group) > 1]\n",
    "\n",
    "def safe_move(file_path, target_dir):\n",
    "    \"\"\"安全移动文件，处理文件名冲突\"\"\"\n",
    "    filename = os.path.basename(file_path)\n",
    "    dest_path = os.path.join(target_dir, filename)\n",
    "    \n",
    "    if not os.path.exists(dest_path):\n",
    "        shutil.move(file_path, dest_path)\n",
    "        print(f\"已移动: {file_path} -> {dest_path}\")\n",
    "        return\n",
    "    \n",
    "    base, ext = os.path.splitext(filename)\n",
    "    counter = 1\n",
    "    while True:\n",
    "        new_name = f\"{base}_dup{counter}{ext}\"\n",
    "        new_dest = os.path.join(target_dir, new_name)\n",
    "        if not os.path.exists(new_dest):\n",
    "            shutil.move(file_path, new_dest)\n",
    "            print(f\"已移动（重命名）: {file_path} -> {new_dest}\")\n",
    "            return\n",
    "        counter += 1\n",
    "\n",
    "def main():\n",
    "    # 配置参数\n",
    "    source_dir = \"\"      # 要检查的源目录\n",
    "    target_dir = \"\"  # 存放重复文件的目录\n",
    "    \n",
    "    # 确保目标目录存在\n",
    "    os.makedirs(target_dir, exist_ok=True)\n",
    "    \n",
    "    # 查找重复项\n",
    "    duplicates = find_duplicates(source_dir)\n",
    "    \n",
    "    if not duplicates:\n",
    "        print(\"未找到重复文件\")\n",
    "        return\n",
    "    \n",
    "    # 移动重复文件\n",
    "    total_moved = 0\n",
    "    for group in duplicates:\n",
    "        print(f\"发现重复文件组（{len(group)} 个文件）:\")\n",
    "        for file_path in group:\n",
    "            print(f\"  - {file_path}\")\n",
    "            safe_move(file_path, target_dir)\n",
    "            total_moved += 1\n",
    "    \n",
    "    print(f\"\\n完成！共移动 {total_moved} 个重复文件到 {target_dir}\")\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    main()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
